{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Format DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(700, 21)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.542875</td>\n",
       "      <td>1.012219</td>\n",
       "      <td>0.529826</td>\n",
       "      <td>0.096133</td>\n",
       "      <td>1.040699</td>\n",
       "      <td>-0.807561</td>\n",
       "      <td>-0.342680</td>\n",
       "      <td>0.858858</td>\n",
       "      <td>0.147749</td>\n",
       "      <td>-0.107334</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.616889</td>\n",
       "      <td>-1.454160</td>\n",
       "      <td>-0.201290</td>\n",
       "      <td>0.863571</td>\n",
       "      <td>0.437816</td>\n",
       "      <td>0.127156</td>\n",
       "      <td>1.589943</td>\n",
       "      <td>0.774433</td>\n",
       "      <td>0.099306</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.704136</td>\n",
       "      <td>0.725639</td>\n",
       "      <td>-0.811169</td>\n",
       "      <td>1.068405</td>\n",
       "      <td>0.056545</td>\n",
       "      <td>0.118662</td>\n",
       "      <td>-0.425186</td>\n",
       "      <td>0.668945</td>\n",
       "      <td>0.081395</td>\n",
       "      <td>-0.225007</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.382115</td>\n",
       "      <td>1.202129</td>\n",
       "      <td>-0.940534</td>\n",
       "      <td>0.758537</td>\n",
       "      <td>-0.313684</td>\n",
       "      <td>-1.114822</td>\n",
       "      <td>1.614120</td>\n",
       "      <td>-0.782022</td>\n",
       "      <td>-2.004021</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.649923</td>\n",
       "      <td>-1.174697</td>\n",
       "      <td>-1.001881</td>\n",
       "      <td>-1.332309</td>\n",
       "      <td>-1.215187</td>\n",
       "      <td>0.270599</td>\n",
       "      <td>-2.110571</td>\n",
       "      <td>-0.462214</td>\n",
       "      <td>-0.207160</td>\n",
       "      <td>0.634851</td>\n",
       "      <td>...</td>\n",
       "      <td>0.704766</td>\n",
       "      <td>0.268407</td>\n",
       "      <td>0.280990</td>\n",
       "      <td>-0.139143</td>\n",
       "      <td>0.410571</td>\n",
       "      <td>0.060985</td>\n",
       "      <td>0.031953</td>\n",
       "      <td>-0.403103</td>\n",
       "      <td>-0.293248</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.661594</td>\n",
       "      <td>-0.053371</td>\n",
       "      <td>-1.867264</td>\n",
       "      <td>1.872130</td>\n",
       "      <td>0.207626</td>\n",
       "      <td>0.000256</td>\n",
       "      <td>0.161002</td>\n",
       "      <td>0.092727</td>\n",
       "      <td>-0.376592</td>\n",
       "      <td>-0.312441</td>\n",
       "      <td>...</td>\n",
       "      <td>1.079768</td>\n",
       "      <td>0.963923</td>\n",
       "      <td>0.864046</td>\n",
       "      <td>1.094562</td>\n",
       "      <td>-0.861178</td>\n",
       "      <td>0.254324</td>\n",
       "      <td>0.459350</td>\n",
       "      <td>-0.577521</td>\n",
       "      <td>0.658839</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.242715</td>\n",
       "      <td>0.981300</td>\n",
       "      <td>0.399682</td>\n",
       "      <td>-1.361206</td>\n",
       "      <td>1.865577</td>\n",
       "      <td>0.355011</td>\n",
       "      <td>-0.043375</td>\n",
       "      <td>1.075501</td>\n",
       "      <td>0.225397</td>\n",
       "      <td>-1.153797</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.890420</td>\n",
       "      <td>-1.206708</td>\n",
       "      <td>-0.645250</td>\n",
       "      <td>0.978400</td>\n",
       "      <td>-1.146337</td>\n",
       "      <td>1.830191</td>\n",
       "      <td>-0.218601</td>\n",
       "      <td>-1.670150</td>\n",
       "      <td>-0.024537</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6  \\\n",
       "0  1.542875  1.012219  0.529826  0.096133  1.040699 -0.807561 -0.342680   \n",
       "1 -0.704136  0.725639 -0.811169  1.068405  0.056545  0.118662 -0.425186   \n",
       "2  1.649923 -1.174697 -1.001881 -1.332309 -1.215187  0.270599 -2.110571   \n",
       "3 -1.661594 -0.053371 -1.867264  1.872130  0.207626  0.000256  0.161002   \n",
       "4  1.242715  0.981300  0.399682 -1.361206  1.865577  0.355011 -0.043375   \n",
       "\n",
       "          7         8         9 ...        11        12        13        14  \\\n",
       "0  0.858858  0.147749 -0.107334 ... -0.616889 -1.454160 -0.201290  0.863571   \n",
       "1  0.668945  0.081395 -0.225007 ... -0.382115  1.202129 -0.940534  0.758537   \n",
       "2 -0.462214 -0.207160  0.634851 ...  0.704766  0.268407  0.280990 -0.139143   \n",
       "3  0.092727 -0.376592 -0.312441 ...  1.079768  0.963923  0.864046  1.094562   \n",
       "4  1.075501  0.225397 -1.153797 ... -0.890420 -1.206708 -0.645250  0.978400   \n",
       "\n",
       "         15        16        17        18        19  y  \n",
       "0  0.437816  0.127156  1.589943  0.774433  0.099306  0  \n",
       "1 -0.313684 -1.114822  1.614120 -0.782022 -2.004021  0  \n",
       "2  0.410571  0.060985  0.031953 -0.403103 -0.293248  1  \n",
       "3 -0.861178  0.254324  0.459350 -0.577521  0.658839  1  \n",
       "4 -1.146337  1.830191 -0.218601 -1.670150 -0.024537  0  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.datasets import make_classification\n",
    "\n",
    "x, y = make_classification(n_samples=700, n_classes=2, shuffle=True, random_state=32)\n",
    "train_df = pd.DataFrame(x, columns=range(x.shape[1]))\n",
    "train_df[\"y\"] = y\n",
    "\n",
    "print(train_df.shape)\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set Up Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-Experiment Key:   'QVqPKuVZbfho-zk60rJMGsmbpahcFa1Po6hf9aVPWFc='\n"
     ]
    }
   ],
   "source": [
    "from hyperparameter_hunter import Environment, CVExperiment\n",
    "from sklearn.model_selection import RepeatedStratifiedKFold\n",
    "\n",
    "env = Environment(\n",
    "    train_dataset=train_df,\n",
    "    results_path=\"HyperparameterHunterAssets\",\n",
    "    target_column=\"y\",\n",
    "    metrics=[\"hamming_loss\"],\n",
    "    cv_type=RepeatedStratifiedKFold,\n",
    "    cv_params=dict(n_repeats=2, n_splits=10, random_state=1337),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that HyperparameterHunter has an active `Environment`, we can do two things:\n",
    "\n",
    "# 1. Perform Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<20:51:47> Validated Environment:  'QVqPKuVZbfho-zk60rJMGsmbpahcFa1Po6hf9aVPWFc='\n",
      "<20:51:47> Initialized Experiment: '15b9efd6-0fbc-4796-8627-5d25b817c443'\n",
      "<20:51:47> Hyperparameter Key:     '-IXQQcLRa5eTe7vPDpD9l7gsq-zO73d7r2FPZ9Bt6cI='\n",
      "<20:51:47> \n",
      "<20:51:47> \n",
      "<20:51:47> F0.0 AVG:   OOF(hamming_loss=0.09859)  |  Time Elapsed: 0.53661 s\n",
      "<20:51:48> F0.1 AVG:   OOF(hamming_loss=0.12676)  |  Time Elapsed: 0.53318 s\n",
      "<20:51:49> F0.2 AVG:   OOF(hamming_loss=0.04225)  |  Time Elapsed: 0.5238 s\n",
      "<20:51:49> F0.3 AVG:   OOF(hamming_loss=0.05714)  |  Time Elapsed: 0.57586 s\n",
      "<20:51:50> F0.4 AVG:   OOF(hamming_loss=0.08571)  |  Time Elapsed: 0.52476 s\n",
      "<20:51:50> F0.5 AVG:   OOF(hamming_loss=0.05714)  |  Time Elapsed: 0.52243 s\n",
      "<20:51:51> F0.6 AVG:   OOF(hamming_loss=0.07143)  |  Time Elapsed: 0.55067 s\n",
      "<20:51:51> F0.7 AVG:   OOF(hamming_loss=0.07246)  |  Time Elapsed: 0.52484 s\n",
      "<20:51:52> F0.8 AVG:   OOF(hamming_loss=0.10145)  |  Time Elapsed: 0.53572 s\n",
      "<20:51:52> F0.9 AVG:   OOF(hamming_loss=0.05797)  |  Time Elapsed: 0.52639 s\n",
      "<20:51:52> Repetition 0 AVG:   OOF(hamming_loss=0.07714)  |  Time Elapsed: 5.36572 s\n",
      "<20:51:52> \n",
      "<20:51:53> F1.0 AVG:   OOF(hamming_loss=0.05634)  |  Time Elapsed: 0.52407 s\n",
      "<20:51:53> F1.1 AVG:   OOF(hamming_loss=0.04225)  |  Time Elapsed: 0.53208 s\n",
      "<20:51:54> F1.2 AVG:   OOF(hamming_loss=0.04225)  |  Time Elapsed: 0.52469 s\n",
      "<20:51:54> F1.3 AVG:   OOF(hamming_loss=0.07143)  |  Time Elapsed: 0.52134 s\n",
      "<20:51:55> F1.4 AVG:   OOF(hamming_loss=0.12857)  |  Time Elapsed: 0.52993 s\n",
      "<20:51:55> F1.5 AVG:   OOF(hamming_loss=0.08571)  |  Time Elapsed: 0.53854 s\n",
      "<20:51:56> F1.6 AVG:   OOF(hamming_loss=0.08571)  |  Time Elapsed: 0.51882 s\n",
      "<20:51:57> F1.7 AVG:   OOF(hamming_loss=0.08696)  |  Time Elapsed: 0.52756 s\n",
      "<20:51:57> F1.8 AVG:   OOF(hamming_loss=0.04348)  |  Time Elapsed: 0.53746 s\n",
      "<20:51:58> F1.9 AVG:   OOF(hamming_loss=0.13043)  |  Time Elapsed: 0.52844 s\n",
      "<20:51:58> Repetition 1 AVG:   OOF(hamming_loss=0.07714)  |  Time Elapsed: 5.29512 s\n",
      "<20:51:58> \n",
      "<20:51:58> FINAL:    OOF(hamming_loss=0.07857)  |  Time Elapsed: 10.66891 s\n",
      "<20:51:58> \n",
      "<20:51:58> Saving results for Experiment: '15b9efd6-0fbc-4796-8627-5d25b817c443'\n"
     ]
    }
   ],
   "source": [
    "from rgf import RGFClassifier\n",
    "\n",
    "experiment = CVExperiment(\n",
    "    model_initializer=RGFClassifier,\n",
    "    model_init_params=dict(max_leaf=1000, algorithm='RGF', min_samples_leaf=10),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Hyperparameter Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validated Environment with key: \"QVqPKuVZbfho-zk60rJMGsmbpahcFa1Po6hf9aVPWFc=\"\n",
      "\u001b[31mSaved Result Files\u001b[0m\n",
      "\u001b[31m______________________________________________________________________________________________________________\u001b[0m\n",
      " Step |       ID |   Time |      Value |   algorithm |        l2 |   learning_rate |      loss |   normalize | \n",
      "Experiments matching cross-experiment key/algorithm: 1\n",
      "Experiments fitting in the given space: 1\n",
      "Experiments matching current guidelines: 1\n",
      "    0 | 15b9efd6 | 00m00s | \u001b[35m   0.07857\u001b[0m | \u001b[32m        RGF\u001b[0m | \u001b[32m   0.1000\u001b[0m | \u001b[32m         0.5000\u001b[0m | \u001b[32m      Log\u001b[0m | \u001b[32m          0\u001b[0m | \n",
      "\u001b[31mHyperparameter Optimization\u001b[0m\n",
      "\u001b[31m______________________________________________________________________________________________________________\u001b[0m\n",
      " Step |       ID |   Time |      Value |   algorithm |        l2 |   learning_rate |      loss |   normalize | \n",
      "    1 | f9f825df | 00m12s | \u001b[35m   0.07714\u001b[0m | \u001b[32m    RGF_Sib\u001b[0m | \u001b[32m   0.1710\u001b[0m | \u001b[32m         0.5947\u001b[0m | \u001b[32m     Expo\u001b[0m | \u001b[32m          0\u001b[0m | \n",
      "    2 | abf46bd8 | 00m12s | \u001b[35m   0.07571\u001b[0m | \u001b[32m    RGF_Sib\u001b[0m | \u001b[32m   0.2793\u001b[0m | \u001b[32m         0.5386\u001b[0m | \u001b[32m       LS\u001b[0m | \u001b[32m          0\u001b[0m | \n",
      "    3 | 68d40c54 | 00m10s |    0.07714 |         RGF |    0.2022 |          0.5206 |      Expo |           1 | \n",
      "    4 | ce25d019 | 00m13s |    0.08143 |     RGF_Sib |    0.0245 |          0.6065 |       Log |           0 | \n",
      "    5 | dca509e6 | 00m12s | \u001b[35m   0.07429\u001b[0m | \u001b[32m    RGF_Sib\u001b[0m | \u001b[32m   0.2717\u001b[0m | \u001b[32m         0.4808\u001b[0m | \u001b[32m       LS\u001b[0m | \u001b[32m          1\u001b[0m | \n",
      "    6 | 0380f1d2 | 00m12s |    0.07429 |     RGF_Sib |    0.2157 |          0.6243 |        LS |           1 | \n",
      "    7 | 0c1a98a1 | 00m10s |    0.08000 |         RGF |    0.0583 |          0.3180 |       Log |           1 | \n",
      "    8 | e061ca32 | 00m11s | \u001b[35m   0.07286\u001b[0m | \u001b[32m        RGF\u001b[0m | \u001b[32m   0.2369\u001b[0m | \u001b[32m         0.4316\u001b[0m | \u001b[32m      Log\u001b[0m | \u001b[32m          0\u001b[0m | \n",
      "    9 | fa22344f | 00m15s |    0.07857 |     RGF_Opt |    0.0966 |          0.6939 |       Log |           1 | \n",
      "   10 | 32a1a3b4 | 00m08s |    0.07286 |         RGF |    0.3000 |          0.3000 |        LS |           1 | \n",
      "Optimization loop completed in 0:01:58.690922\n",
      "Best score was 0.07285714285714286 from Experiment \"e061ca32-5e80-4b58-998f-97f5c240823d\"\n"
     ]
    }
   ],
   "source": [
    "from hyperparameter_hunter import BayesianOptPro, Real, Integer, Categorical\n",
    "\n",
    "optimizer = BayesianOptPro(iterations=10, random_state=42)\n",
    "\n",
    "optimizer.forge_experiment(\n",
    "    model_initializer=RGFClassifier,\n",
    "    model_init_params=dict(\n",
    "        max_leaf=1000,\n",
    "        algorithm=Categorical(['RGF', 'RGF_Opt', 'RGF_Sib']),\n",
    "        l2=Real(0.01, 0.3),\n",
    "        normalize=Categorical([True, False]),\n",
    "        learning_rate=Real(0.3, 0.7),\n",
    "        loss=Categorical(['LS', 'Expo', 'Log', 'Abs'])\n",
    "    ),\n",
    ")\n",
    "\n",
    "optimizer.go()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice, `optimizer` recognizes our earlier `experiment`'s hyperparameters fit inside the search space/guidelines set for `optimizer`.\n",
    "\n",
    "Then, when optimization is started, it automatically learns from `experiment`'s results - without any extra work for us!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
